#coding:utf-8

ES_NAME = "elasticsearch_grup_data"
ES_URI = "10.4.16.206"
ES_PORT = 9250
ES_INDEX = "cut_data"
ES_TYPE = "cut_data"

from store import EsStore
import json
import datetime

import math

def process1():

    es = EsStore(ES_URI,ES_PORT,ES_INDEX,ES_TYPE)

    T1 = datetime.datetime(2014,6,11)
    T2 = datetime.datetime(2014,6,12)

    res = es.getByTime(T1,T2)
    print res

    filename = 'data.txt'

    f = open(filename,mode='w')

    res = es.next(res['si'])
    while len(res['rs']) > 0:
        print len(res['rs'])
        for r in res['rs']:
            r['term_score'] = json.loads(r['term_score'])
            line = json.dumps(r,ensure_ascii=False)
            f.write(line.encode('utf-8')+'\n')
        res = es.next(res['si'])
    f.close()


def initUS(filename):
    U = dict()
    with open(filename) as f:
        for line in f:
            _,vs,zs = json.loads(line)
            for v in vs:
                U[v] = list(set(zs))
    return U

def process2():

    filename = 'US.5.txt'
    U = initUS(filename)

    filename1 = 'data.txt'
    filename2 = 'data.1.txt'

    f1 = open(filename1,mode='r')
    f2 = open(filename2,mode='w')

    for line in f1:
        m = json.loads(line)
        title = m['cut_title']
        content = m['cut_content']
        if len(content) < 100:
            continue
        i = m['name']
        url = m['url']

        vs = m['term_score']
        vs = [ (v,s) for v,s in vs.items() if s > 8.0 ]

        zs = list()
        for v,s in vs:
            try:
                U[v]
                #zs.extend(U[v])
                zs.append((v,U[v]))
            except:
                pass

        if len(zs) > 1:
            line = json.dumps((i,zs,title,content,url),ensure_ascii=False)
            f2.write(line.encode('utf-8')+'\n')

    f1.close()
    f2.close()


def initMeasure(filename):
    M = dict()
    with open(filename) as f:
        for line in f:
            v,c = line.strip().split()
            v = v.decode('utf-8')
            c = int(c)
            M[v] = c
    return M
CC = 6000000.0
def score(M,title,content):
    m = dict()
    line = (title+" ")*3
    vs = line.strip().split()
    vs.extend(content.strip().split())
    for v in vs:
        c = m.get(v,0)
        m[v] = c+1
    for v,c in m.items():
        s = math.log(c+1.0)*math.log(CC/M.get(v,100.0))
        m[v] = s
    return m

def process3():

    filename = 'measure.term.txt'
    M = initMeasure(filename)

    filename1 = 'data.1.txt'
    filename2 = 'data.2.txt'

    f1 = open(filename1,mode='r')
    f2 = open(filename2,mode='w')

    for line in f1:
        i,zs,title,content,url = json.loads(line)
        title = title.lower()
        content = content.lower()
        m = score(M,title,content)
        m = {v:s for v,s in m.items() if len(v) > 1}
        line = json.dumps((i,zs,m,url),ensure_ascii=False)
        f2.write(line.encode('utf-8')+'\n')

    f1.close()
    f2.close()

if __name__ == '__main__':

    print 'prepare'


    #process1()
    #process2()
    process3()
    
